package com.house.twohouse.config;



import com.house.twohouse.entity.RecordEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * <dependency>
 *             <groupId>org.jsoup</groupId>
 *             <artifactId>jsoup</artifactId>
 *             <version>1.13.1</version>
 *         </dependency>
 */
//爬虫
@Component
public class CrawlerConfig {

    //二手房源
    public List<RecordEntity> secondCrawlerAdd() throws Exception {
        //要爬取网站的url地址      这里爬取贝壳的二手房网页的数据
        String url = "https://sh.ke.com/ershoufang/rs/";
        //解析爬到的网页数据   parse方法返回Document对象
        Document document = Jsoup.parse(new URL(url), 30000);
        //根据对象获取要爬的模块  再根据id找出相应的对象
        Element elementById = document.getElementById("beike");
        //获取所有的li标签
        Elements elements = elementById.getElementsByTag("li");
        ArrayList<RecordEntity> list = new ArrayList<>();
        //循环遍历所有li标签中的元素
        for (Element el : elements) {
            String img = el.getElementsByClass("lj-lazy").eq(0).attr("data-original");
            //System.out.println("爬的图片："+img);
            String title = el.getElementsByClass("VIEWDATA CLICKDATA maidian-detail").eq(0).text();
            //System.out.println("标题信息:"+title);
            String address = el.getElementsByClass("positionInfo").eq(0).text();
            //System.out.println("地理位置:"+location);
            String houseInfo = el.getElementsByClass("houseInfo").eq(0).text();
            //System.out.println("房屋信息："+houseInfo);
            String priceInfo = el.getElementsByClass("totalPrice totalPrice2").eq(0).text();
            //System.out.println("房屋价格："+priceInfo);
            String unitPrice = el.getElementsByClass("unitPrice").eq(0).text();
            //System.out.println("每平方米价格："+unitPrice);
            RecordEntity recordEntity = new RecordEntity();
            //爬取不为null的数据
            if(StringUtils.isNotEmpty(img) && img!=""){
                //赋值
                recordEntity.setRecordImg(img);
                recordEntity.setRecordTitle(title);
                recordEntity.setRecordAddress(address);
                recordEntity.setRecordHouseinfo(houseInfo);
                recordEntity.setRecordPrice(priceInfo);
                recordEntity.setRecordUnitprice(unitPrice);
                //循环放到对象中
                list.add(recordEntity);
            }
        }
        System.out.println(list);
        //返回集合
        return list;
    }


}
